﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Spider.Entitys.Spider;
using System.Text.RegularExpressions;
using Newtonsoft.Json;
using SpiderCore.Loggers;
using SpiderCore.Scheduler;
using SpiderCore.DB;

namespace SpiderCore
{
    public class Spider : ISpider
    {
        #region 属性 字段
        /// <summary>
        /// 初始化日志类型
        /// </summary>
        private readonly ISpiderLogger log = new SpiderMongoDBLog(typeof(Spider));

        /// <summary>
        /// 数据持久化
        /// </summary>
        private readonly IDB db = new RedisDb();

        /// <summary>
        /// 需要排除的 URL后缀  
        /// 例如 .ico .css .pdf .xml
        /// </summary>
        static List<string> UrlExt = new List<string>()
        {
            ".ico",
            ".css",
            ".xml"
        };

        /// <summary>
        /// URL 分类解析
        /// </summary>
        private List<CustomRuleModel> CustomRuleList = null;

        /// <summary>
        /// 页面抓取深度
        /// </summary>
        private int _depth = 0;

        /// <summary>
        /// 设置页面抓取深度
        /// </summary>
        /// <param name="depth"></param>
        public void SetSpiderDepth(int depth)
        {
            _depth = depth;
        }

        /// <summary>
        /// 设置URL匹配规则
        /// </summary>
        /// <param name="CustomRuleList"></param>
        public void SetCustomRuleList(List<CustomRuleModel> CustomRuleList)
        {
            this.CustomRuleList = CustomRuleList;
        }
        #endregion

        #region 构造函数
        /// <summary>
        /// 爬虫模板初始化
        /// </summary>
        /// <param name="CustomRuleList"></param>
        public Spider(List<CustomRuleModel> CustomRuleList)
        {
            this.CustomRuleList = CustomRuleList;
        }

        public Spider() : this(null)
        {

        }
        #endregion

        /// <summary>
        /// 判断URL是否存在
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public bool Exist(string url)
        {
            var key = db.Get<string>(url, null);
            return string.IsNullOrEmpty(key) == false;
        }

        #region 提取URL内容
        /// <summary>
        /// 提取URL内容
        /// </summary>
        /// <param name="response"></param>
        /// <returns></returns>
        public MessageContext Extract(ResponseModel response)
        {
            return ExtractHtmlResponse(response as HtmlResponseModel);
        }
        #endregion

        #region 解析Html内容
        /// <summary>
        /// 解析Html内容
        /// 
        /// 处理步骤：
        /// 1.解析出全部的A链接href地址
        /// 2.逐个对md5(url)签名,并验证是否已存在
        /// 3.如果不存在，则添加到调度器中
        /// 4.返回 MessageContext 消息
        /// </summary>
        /// <param name="htmlResponse"></param>
        /// <returns></returns>
        private HtmlMessageContext ExtractHtmlResponse(HtmlResponseModel htmlResponse)
        {
            //判断是否提取文档链接
            if (htmlResponse.Request.IsExtractLink)
            {
                ExtractLink(htmlResponse);
            }

            var message = new HtmlMessageContext
            {
                Response = htmlResponse.Clone() as ResponseModel,
                Text = htmlResponse.Html,
            };
            //message.Response.Html = htmlResponse.Html;
            return message;
        }
        #endregion

        #region 提取文档链接
        /// <summary>
        /// 提取文档链接
        /// </summary>
        /// <param name="htmlResponse"></param>
        private void ExtractLink(HtmlResponseModel htmlResponse)
        {
            #region 判断URL深度 根据配置决定是否继续抓取
            if (_depth != 0 && _depth <= htmlResponse.Request.Depth)
            {
                return;
            }
            #endregion

            #region URL 处理
            //解析出全部Url并去除重复记录
            var urlList = GetHrefUrlList(htmlResponse.Html, htmlResponse.Request.Url).Distinct();

            //过滤第三方网站域名
            if (htmlResponse.Request.IgnoreThirdpartyDomain)
            {
                var uriHost = new Uri(htmlResponse.Request.Url).Host.Replace("www.", "");

                urlList = urlList.Where(p => p.Contains(uriHost)).ToList();
            }
            #endregion

            //循环处理url列表
            foreach (var url in urlList)
            {
                try
                {
                    var urlHash = url.ToMD5_UTF8();
                    var redisKey = string.Format("spider:url:{0}", urlHash);

                    //是否满足抓取条件
                    if (Exist(redisKey) == false)
                    {
                        var requestUri = new Uri(url);
                        var request = htmlResponse.Request.Clone() as RequestModel;

                        if (request != null && requestUri.IsFile == false)
                        {
                            request.Url = url;
                            request.Referer = htmlResponse.Request.Url;
                            request.Depth = ++htmlResponse.Request.Depth;

                            request = UrlClassify(request, url);

                            //推送到调度队列
                            SchedulerManage.Instance.Push(request);

                            var redisValue = JsonConvert.SerializeObject(request);
                            //推送到缓存  URL判重
                            db.Set(redisKey, redisValue);
                        }
                    }
                }
                catch (Exception exception)
                {
                    log.Error(exception, url + "--" + "SpiderCore.Spider.ExtractLink");
                }
            }
        }
        #endregion

        #region URL分类
        /// <summary>
        /// URL分类
        /// </summary>
        /// <param name="request">当前的请求模型</param>
        /// <returns></returns>
        private RequestModel UrlClassify(RequestModel request, string url)
        {
            if (CustomRuleList != null)
            {
                CustomRuleList = CustomRuleList.OrderBy(i => i.Seq).ToList();
                foreach (var item in CustomRuleList)
                {
                    if (Regex.IsMatch(url, item.UrlAnalysis))
                    {
                        request.CustomRule = item;
                        return request;
                    }
                }
            }
            return request;
        }
        #endregion

        #region 链接正则：href="[\s\S\W]+?"
        /// <summary>
        /// 链接正则：href="[\s\S\W]+?"
        /// </summary>
        /// <param name="html"></param>
        /// <param name="rootUrl"></param>
        /// <returns></returns>
        private static IEnumerable<string> GetHrefUrlList(string html, string rootUrl)
        {
            if (string.IsNullOrEmpty(html)) return new List<string>();
            var uri = new Uri(rootUrl);
            var result = new List<string>();
            var matchCollection = Regex.Matches(html, "href=\"[\\s\\S\\W]+?\"", RegexOptions.IgnoreCase);
            for (var i = 0; i < matchCollection.Count; i++)
            {
                var href = matchCollection[i].Value;

                if (href.Length < 6) continue;

                bool flag = true;
                foreach (var item in UrlExt)
                {
                    if (href.Contains(item))
                    {
                        flag = false;
                        break;
                    }
                }
                if (!flag)
                {
                    continue;
                }

                ///需要把一些无用后缀的URL请求给屏蔽了

                href = href.Substring(6, href.Length - 7);

                if (href.StartsWith("#", StringComparison.CurrentCultureIgnoreCase) || href == "/") continue;

                if (href.StartsWith("javascript:", StringComparison.CurrentCultureIgnoreCase)) continue;

                if (href.StartsWith("//", StringComparison.CurrentCultureIgnoreCase))
                {
                    href = String.Format("http:{0}", href);

                    result.Add(href);
                }

                if (href.StartsWith("/", StringComparison.CurrentCultureIgnoreCase))
                {
                    //补http协议

                    href = String.Format("{0}://{1}/{2}", uri.Scheme, uri.Host, href.TrimStart('/'));

                    result.Add(href);

                    continue;
                }

                if (href.StartsWith("http://", StringComparison.CurrentCultureIgnoreCase) ||
                    href.StartsWith("https://", StringComparison.CurrentCultureIgnoreCase))
                {
                    //直接返回

                    result.Add(href);

                    continue;
                }
                //other
            }
            return result;
        }
        #endregion


    }
}
